from threading import Thread

import redis

# from spider_redis_weijianwei import Spider
from spider_new_keys import Spider

from settings import  HOST,  PWD, PORT, DB

from redisUtils_k import RDSUtill

def connect_redis():
    global db_cn
    host = HOST
    pwd = PWD
    port = PORT
    db = DB
    try:
        db_cn = redis.Redis(host=host, port=port, db=db, decode_responses=True, password=pwd)
    except Exception as e:
        print(e)
    return db_cn


def get_task(i):
    """取任务"""
    global spider
    COMPANY_NAME = 'WeiJianWei'
    while True:
        redis_db = connect_redis()
        pipe = redis_db.pipeline()
        # 自己的储存名称
        pipe.spop(COMPANY_NAME)
        task = pipe.execute()[0]
        if not task:
            print('task finish!')
        # 采集运行函数，通过返回值判断该任务是否成功，不成功则重新扔回任务队列
        name, city = task.split('\u0001')

        spider = Spider(i)
        result = spider.start(name=name,prov=city)
        if result:
            print(name, city)
        else:
            redis_db = connect_redis()
            pipe = redis_db.pipeline()
            pipe.sadd(COMPANY_NAME, task)
            pipe.execute()


class Runner(Thread):
    def __init__(self, i):
        Thread.__init__(self)
        self.i = i
        self.r = RDSUtill()

    def run(self):
        get_task(i)


if __name__ == '__main__':
    thread_li = []
    for i in range(20):
        runner = Runner(i)
        thread_li.append(runner)

    for i in thread_li:
        i.start()

    for i in thread_li:
        i.join()
